import scrapy

from xiaoshuo.items import XiaoshuoItem


class renwokan(scrapy.Spider):
    name = "renwokan"
    start_urls = {
        'https://www.r5k.com/list/10/1/'
    }

    custom_settings = {
        "USER_AGENT": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36",
    }

    def parse(self, response):
        item = XiaoshuoItem()
        item['url'] = response.xpath('/html/body/div[3]/div[2]/div/div/div[1]/a/@href').extract()
        item['name'] = response.xpath('/html/body/div[3]/div[2]/div/div/div[2]/div[1]/a/text()').extract()
        jianjies = response.xpath('/html/body/div[3]/div[2]/div/div/div[2]/div[3]/text()').extract()
        for i in range(len(jianjies)):
            jianjies[i] = jianjies[i].replace("\r\n","")
            jianjies[i] = jianjies[i].replace(" ","")
            jianjies[i] = jianjies[i].replace("\u3000","")
        item['jianjie'] = jianjies
        for i in range(len(item['url'])):
            item['url'][i] = 'https://www.r5k.com' + item['url'][i]
        # print(jianjies)
        # print(item)
        yield item
        next_page = response.xpath('/html/body/div[4]/a[13]/@href')[0].extract()
        next_page = 'https://www.r5k.com'+next_page
        if next_page is not None:
            next_page = response.urljoin(next_page)
            yield scrapy.Request(next_page,callback= self.parse)